<?php
function getContentURL($url){
    do{
        $status = false;
        $ch=curl_init();
        curl_setopt($ch,CURLOPT_URL,$url);
        curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
        curl_setopt($ch, CURLOPT_REFERER, 'http://www.google.com');
        //curl_setopt($ch, CURLOPT_PROXY, '91.75.24.162:3128');//proxy fake ip truy cap
        curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 120);
        curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.2) Gecko/20090729 Firefox/3.5.2 GTB5');
        //curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
        curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
        $result = curl_exec($ch);
        //var_dump($result);
        $info = curl_getinfo($ch);
        //var_dump($info);
        if($info['http_code'] == 301){
            $status = true;
            /*echo '<pre>';
            var_dump(parse_url($url));
            echo '</pre>';die;*/
            $domain = parse_url($url);
            $domain = $domain['scheme'].'://'.$domain['host'];
            $info = get_headers($url);
            $url = str_replace('Location:','',$info[11]);
            $url = str_replace('location:','',$url);
            $url = $domain.trim($url);
            //echo $url,'<br>';
        }
    }while($status);
    curl_close($ch);
    //return array('content' => $result,'http_code' => $info['http_code']);
    return $result;
}


function getOG($sites_html){
    $html = new DOMDocument();
    @$html->loadHTML($sites_html);
    $meta_og_img = null;
    $meta_og_title = null;
    foreach($html->getElementsByTagName('meta') as $meta) {
        if($meta->getAttribute('property')=='og:image'){
            $tags['image'] = $meta->getAttribute('content');
        }
        if($meta->getAttribute('property')=='og:title'){
            $tags['title'] = $meta->getAttribute('content');
        }
    }
    return $tags;
}

function preDump($data,$die=false){
    echo '<pre>';
    var_dump($data);
    echo '</pre>';
    if($die) die;
}

function esNoTag($data){
    return trim(html_entity_decode(strip_tags($data), ENT_NOQUOTES, "UTF-8"));
}

function getFullLink($url,$link){
    return (stripos($link,"http")===false) ? $url.'/'.$link : $link;
}

function clean($string) {
    $string = str_replace(" ", "-", $string); // Replaces all spaces with hyphens.
    $string = preg_replace('/[^A-Za-z0-9\-]/', '', $string); // Removes special chars.

    return preg_replace('/-+/', '-', $string); // Replaces multiple hyphens with single one.
}

function stripUnicode($str){
    if(!$str) return false;
    $trans = array(
        "đ"=>"d","ă"=>"a","â"=>"a","á"=>"a","à"=>"a","ả"=>"a","ã"=>"a","ạ"=>"a",
        "ấ"=>"a","ầ"=>"a","ẩ"=>"a","ẫ"=>"a","ậ"=>"a",
        "ắ"=>"a","ằ"=>"a","ẳ"=>"a","ẵ"=>"a","ặ"=>"a",
        "é"=>"e","è"=>"e","ẻ"=>"e","ẽ"=>"e","ẹ"=>"e",
        "ế"=>"e","ề"=>"e","ể"=>"e","ễ"=>"e","ệ"=>"e",
        "í"=>"i","ì"=>"i","ỉ"=>"i","ĩ"=>"i","ị"=>"i",
        "ư"=>"u","ô"=>"o","ơ"=>"o","ê"=>"e",
        "Ư"=>"u","Ô"=>"o","Ơ"=>"o","Ê"=>"e",
        "ú"=>"u","ù"=>"u","ủ"=>"u","ũ"=>"u","ụ"=>"u",
        "ứ"=>"u","ừ"=>"u","ử"=>"u","ữ"=>"u","ự"=>"u",
        "ó"=>"o","ò"=>"o","ỏ"=>"o","õ"=>"o","ọ"=>"o",
        "ớ"=>"o","ờ"=>"o","ở"=>"o","ỡ"=>"o","ợ"=>"o",
        "ố"=>"o","ồ"=>"o","ổ"=>"o","ỗ"=>"o","ộ"=>"o",
        "ú"=>"u","ù"=>"u","ủ"=>"u","ũ"=>"u","ụ"=>"u",
        "ứ"=>"u","ừ"=>"u","ử"=>"u","ữ"=>"u","ự"=>"u",'ý'=>'y','ỳ'=>'y','ỷ'=>'y','ỹ'=>'y','ỵ'=>'y', 'Ý'=>'Y','Ỳ'=>'Y','Ỷ'=>'Y','Ỹ'=>'Y','Ỵ'=>'Y',
        "Đ"=>"D","Ă"=>"A","Â"=>"A","Á"=>"A","À"=>"A","Ả"=>"A","Ã"=>"A","Ạ"=>"A",
        "Ấ"=>"A","Ầ"=>"A","Ẩ"=>"A","Ẫ"=>"A","Ậ"=>"A",
        "Ắ"=>"A","Ằ"=>"A","Ẳ"=>"A","Ẵ"=>"A","Ặ"=>"A",
        "É"=>"E","È"=>"E","Ẻ"=>"E","Ẽ"=>"E","Ẹ"=>"E",
        "Ế"=>"E","Ề"=>"E","Ể"=>"E","Ễ"=>"E","Ệ"=>"E",
        "Í"=>"I","Ì"=>"I","Ỉ"=>"I","Ĩ"=>"I","Ị"=>"I",
        "Ư"=>"U","Ô"=>"O","Ơ"=>"O","Ê"=>"E",
        "Ư"=>"U","Ô"=>"O","Ơ"=>"O","Ê"=>"E",
        "Ú"=>"U","Ù"=>"U","Ủ"=>"U","Ũ"=>"U","Ụ"=>"U",
        "Ứ"=>"U","Ừ"=>"U","Ử"=>"U","Ữ"=>"U","Ự"=>"U",
        "Ó"=>"O","Ò"=>"O","Ỏ"=>"O","Õ"=>"O","Ọ"=>"O",
        "Ớ"=>"O","Ờ"=>"O","Ở"=>"O","Ỡ"=>"O","Ợ"=>"O",
        "Ố"=>"O","Ồ"=>"O","Ổ"=>"O","Ỗ"=>"O","Ộ"=>"O",
        "Ú"=>"U","Ù"=>"U","Ủ"=>"U","Ũ"=>"U","Ụ"=>"U",
        "Ứ"=>"U","Ừ"=>"U","Ử"=>"U","Ữ"=>"U","Ự"=>"U","-"=>"",":"=>"");
    $str = strtr($str, $trans);
    $str = preg_replace(array('/\s+/','/[^A-Za-z0-9\-]/'), array('-',''), $str);
    return strtolower($str);
}

function insCrawlerCC($cnf){
    $htmlList = str_get_html(strip_tags(getContentURL($cnf['link']),'<h2><a>'));
    foreach( $htmlList->find('.question-title a') as $k => $e){
        $link = getFullLink($cnf['url'],$e->href);
        echo $link.'<br>';
        $htmlDetail = str_get_html(getContentURL($link));
        $data['type'] =  MySQL::SQLValue('Q');
        $data['categoryid'] = 1;
        $data['catidpath1'] = 1;
        $data['userid'] = 1;
        $data['createip'] = 2130706433;
        $data['created'] =  MySQL::SQLValue(date('Y-m-d H:i:s'));
        $data['title'] =  esNoTag($htmlDetail->find('.question-title',0)->plaintext);
        $data['content'] =  esNoTag($htmlDetail->find('.question-description',0)->plaintext);
        $data['content'] = $data['content'] ? $data['content'] : $data['title'];
        $data['title'] = MySQL::SQLValue($data['title']);
        $data['content'] = MySQL::SQLValue($data['content']);
        $data['tags'] = '';
        foreach($htmlDetail->find('.question-tags .tag') as $tag){
            $data['tags'] .= esNoTag($tag->plaintext).',';
        }
        if($data['tags'])
            $data['tags'] = substr($data['tags'],0,-1);
        $data['tags'] = MySQL::SQLValue($data['tags']);
        /*foreach($htmlDetail->find('.idea-content') as $rep){
            $data['rep'][] = array('desc'=>'<strong>'.esNoTag($rep->find('.idea-title',0)->plaintext).'<strong><p>'.esNoTag($rep->find('.idea-description',0)->plaintext).'</p>',
                                   'auth'=>esNoTag($rep->find('.idea-author a',0)->plaintext));
        }*/

        inPosts('qa_posts',$data);

        preDump($data,true);
        unset($htmlDetail);
    }
}